package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"time"
)

func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := r.Peek(1024)
	if err != nil {
		return unicode.UTF8
	}

	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}

func Fetch(url string) (content []byte, err error) {
	client := &http.Client{
		Timeout: 10 * time.Second,
	}
	resp, err := client.Get(url)
	if err != nil {
		return nil, fmt.Errorf("Got wrong url: %s, error: %s.\n", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Got wrong url: %s, error: %s, Error: status code %d.\n", url, err, resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	encoding := determineEncoding(bodyReader)
	return ioutil.ReadAll(transform.NewReader(bodyReader, encoding.NewDecoder()))
}
